
from os.path import dirname, join, basename
import importlib
import glob
from multiprocessing import Process, Pool
from Utils.Function import *
from Utils.RedisHelper import RedisHelper
from apscheduler.schedulers.background import BackgroundScheduler
from API.ProxyAPI import *


def CheckProxys():
    while True:
        redis = RedisHelper()
        print('开始检测正式库')
        pool = Pool(processes=5)
        keys = redis.getkeys()
        for key in keys:
            pool.apply_async(CheckProxy,(key,))
        pool.close()
        pool.join()
        print('结束检测正式库')
        time.sleep(60)

def CheckTempProxys():
    while True:
        redis = RedisHelper()
        print('开始检测临时库')
        temppool = Pool(processes=20)
        tempkeys = redis.temp_getkeys()
        for tempkey in tempkeys:
            temppool.apply_async(CheckTempProxy, (tempkey,))
        temppool.close()
        temppool.join()
        print('结束检测临时库')
        time.sleep(60)


def ShowStatus():
    while True:
        redis = RedisHelper()
        keys1 = redis.getkeys()
        keys2 = redis.temp_getkeys()
        print('正式IP：' + str(len(keys1)) + ' 临时IP：' + str(len(keys2)))
        time.sleep(3)


class SpiderJob(object):
    def LoadSpiders(self):
        scheduler = BackgroundScheduler()
        currdir = dirname(__file__)
        for x in glob.glob(join(currdir, 'Spiders/*.py')):
            if not x.startswith('__'):
                clsname = basename(x)[:-3]
                if clsname[-6:] == "Spider":
                    module = 'Spiders.'+clsname
                    obj = importlib.import_module(module)
                    obj = getattr(obj, clsname)
                    spiderclass = obj()
                    getlist = getattr(spiderclass, 'getlist')
                    #getlist()
                    config = getattr(spiderclass, 'config')
                    interval = config['interval']
                    scheduler.add_job(getlist, 'interval', minutes=interval)
        scheduler.start()


if __name__ == '__main__':
    checkproxys = Process(target=CheckProxys)
    checkproxys.start()
    checktemps = Process(target=CheckTempProxys)
    checktemps.start()
    showstatus = Process(target=ShowStatus)
    showstatus.start()

    api = Process(target=runapi)
    api.start()

    job = SpiderJob()
    job.LoadSpiders()

